home *** CD-ROM | disk | FTP | other *** search
/ MacFormat 1995 March / macformat-022.iso / Shareware City / Developers / src / out-of-phase-102-c / OutOfPhase 1.02 Source / OutOfPhase Folder / CompilerScanner.c < prev    next >
Encoding:
C/C++ Source or Header  |  1994-11-23  |  22.3 KB  |  868 lines  |  [TEXT/KAHL]
  1. /* CompilerScanner.c */
  2. /*****************************************************************************/
  3. /*                                                                           */
  4. /*    Out Of Phase:  Digital Music Synthesis on General Purpose Computers    */
  5. /*    Copyright (C) 1994  Thomas R. Lawrence                                 */
  6. /*                                                                           */
  7. /*    This program is free software; you can redistribute it and/or modify   */
  8. /*    it under the terms of the GNU General Public License as published by   */
  9. /*    the Free Software Foundation; either version 2 of the License, or      */
  10. /*    (at your option) any later version.                                    */
  11. /*                                                                           */
  12. /*    This program is distributed in the hope that it will be useful,        */
  13. /*    but WITHOUT ANY WARRANTY; without even the implied warranty of         */
  14. /*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the          */
  15. /*    GNU General Public License for more details.                           */
  16. /*                                                                           */
  17. /*    You should have received a copy of the GNU General Public License      */
  18. /*    along with this program; if not, write to the Free Software            */
  19. /*    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.              */
  20. /*                                                                           */
  21. /*    Thomas R. Lawrence can be reached at tomlaw@world.std.com.             */
  22. /*                                                                           */
  23. /*****************************************************************************/
  24.  
  25. #include "MiscInfo.h"
  26. #include "Audit.h"
  27. #include "Debug.h"
  28. #include "Definitions.h"
  29.  
  30. #include "CompilerScanner.h"
  31. #include "TrashTracker.h"
  32. #include "Memory.h"
  33. #include "DataMunging.h"
  34. #include "StringThing.h"
  35. #include "Numbers.h"
  36.  
  37.  
  38. #define MAXNUMKEYWORDS (128)
  39.  
  40.  
  41. #define OPAREN '('
  42. #define CPAREN ')'
  43. #define OBRACKET '['
  44. #define CBRACKET ']'
  45.  
  46.  
  47. struct TokenRec
  48.     {
  49.         TokenTypes                Type;
  50.         union
  51.             {
  52.                 long                            IntegerValue;
  53.                 double                        DoubleValue;
  54.                 float                            SingleValue;
  55.                 MyBoolean                    BooleanValue;
  56.                 char*                            StringValue;
  57.                 largefixedsigned    FixedValue;
  58.             } u;
  59.     };
  60.  
  61.  
  62. typedef struct
  63.     {
  64.         char*                            KeywordName;
  65.         long                            KeywordLength;
  66.         long                            TagValue;
  67.     } KeywordRec;
  68.  
  69.  
  70. struct ScannerRec
  71.     {
  72.         char*                            Block;
  73.         long                            BlockLength;
  74.         long                            Index;
  75.         long                            LineNumber;
  76.         long                            NumKeywords;
  77.         TrashTrackRec*        Allocator;
  78.         TokenRec*                    PushedBackToken;
  79.         KeywordRec                KeywordList[MAXNUMKEYWORDS];
  80.     };
  81.  
  82.  
  83. /* this type is used when parsing floating point numbers to remember what */
  84. /* part we're on */
  85. typedef enum
  86.     {
  87.         eIntegerPart EXECUTE(= -6751),
  88.         eFractionalPart,
  89.         eExponentialPart,
  90.         eExponNumberPart,
  91.         eNumberFinished
  92.     } NumStateType;
  93.  
  94.  
  95. /* this type is used for explicitly specifying the type of a number */
  96. typedef enum
  97.     {
  98.         eTypeNotSpecified EXECUTE(= -5612),
  99.         eTypeSingle,
  100.         eTypeDouble,
  101.         eTypeFixed,
  102.         eTypeInteger
  103.     } NumFormType;
  104.  
  105.  
  106. /* create a scanner information record */
  107. ScannerRec*                    NewScanner(struct TrashTrackRec* TrashTracker, char* RawData)
  108.     {
  109.         ScannerRec*                Scanner;
  110.  
  111.         CheckPtrExistence(RawData);
  112.  
  113.         Scanner = (ScannerRec*)AllocTrackedBlock(sizeof(ScannerRec),TrashTracker);
  114.         if (Scanner == NIL)
  115.             {
  116.                 return NIL;
  117.             }
  118.         SetTag(Scanner,"ScannerRec");
  119.  
  120.         Scanner->Block = AllocTrackedBlock(PtrSize(RawData),TrashTracker);
  121.         if (Scanner->Block == NIL)
  122.             {
  123.                 return NIL;
  124.             }
  125.         SetTag(Scanner->Block,"ScannerRec:  data block");
  126.  
  127.         CopyData(RawData,Scanner->Block,PtrSize(RawData));
  128.  
  129.         Scanner->BlockLength = PtrSize(Scanner->Block);
  130.         Scanner->Index = 0;
  131.         Scanner->LineNumber = 1;
  132.         Scanner->NumKeywords = 0;
  133.         Scanner->Allocator = TrashTracker;
  134.         Scanner->PushedBackToken = NIL;
  135.  
  136.         return Scanner;
  137.     }
  138.  
  139.  
  140. /* negative = L < R; zero = equal, positive = L > R */
  141. static int                    CompareStrings(char* LRef, long LLen, char* RRef, long RLen)
  142.     {
  143.         long                            Scan;
  144.  
  145.         Scan = 0;
  146.         while ((Scan < LLen) && (Scan < RLen))
  147.             {
  148.                 int                                Result;
  149.  
  150.                 Result = LRef[Scan] - RRef[Scan];
  151.                 if (Result != 0)
  152.                     {
  153.                         /* negative means LRef < RRef */
  154.                         /* positive means LRef > RRef */
  155.                         return Result;
  156.                     }
  157.                 Scan += 1;
  158.             }
  159.         /* if strings are equal up to smallest lengthed one, then */
  160.         /* if left is shorter, then left is less */
  161.         /* if right is shorter, then right is less */
  162.         /* if lengths are equal, then strings are equal */
  163.         return LLen - RLen;
  164.     }
  165.  
  166.  
  167. /* add a token to the scanner.  Keyword is a null terminated statically */
  168. /* allocated string */
  169. void                                AddKeywordToScanner(ScannerRec* Scanner, char* Keyword, long Tag)
  170.     {
  171.         long                            Scan;
  172.         long                            KeywordLength;
  173.  
  174.         CheckPtrExistence(Scanner);
  175.  
  176.         ERROR(Scanner->NumKeywords == MAXNUMKEYWORDS,PRERR(ForceAbort,
  177.             "AddKeywordToScanner:  too many keywords"));
  178.  
  179.         KeywordLength = StrLen(Keyword);
  180.         Scan = 0;
  181.         while (Scan < Scanner->NumKeywords)
  182.             {
  183.                 int                                Compare;
  184.  
  185.                 Compare = CompareStrings(Keyword,KeywordLength,Scanner->KeywordList[Scan]
  186.                     .KeywordName,Scanner->KeywordList[Scan].KeywordLength);
  187.                 ERROR(Compare == 0,PRERR(ForceAbort,
  188.                     "AddKeywordToScanner:  keyword already known"));
  189.                 if (Compare < 0)
  190.                     {
  191.                         long                            Index;
  192.  
  193.                         /* our string is less, so insert at Scan */
  194.                         for (Index = Scanner->NumKeywords; Index > Scan; Index -= 1)
  195.                             {
  196.                                 Scanner->KeywordList[Index] = Scanner->KeywordList[Index - 1];
  197.                             }
  198.                         Scanner->KeywordList[Scan].KeywordName = Keyword;
  199.                         Scanner->KeywordList[Scan].KeywordLength = KeywordLength;
  200.                         Scanner->KeywordList[Scan].TagValue = Tag;
  201.                         Scanner->NumKeywords += 1;
  202.                         return;
  203.                     }
  204.                 /* else go to the next one */
  205.                 Scan += 1;
  206.             }
  207.         /* if there was nowhere to insert, then just put it in there hey */
  208.         Scanner->KeywordList[Scanner->NumKeywords].KeywordName = Keyword;
  209.         Scanner->KeywordList[Scanner->NumKeywords].KeywordLength = KeywordLength;
  210.         Scanner->KeywordList[Scanner->NumKeywords].TagValue = Tag;
  211.         Scanner->NumKeywords += 1;
  212.     }
  213.  
  214.  
  215. /* get the line number (starting at 1) that the scanner is on right now */
  216. long                                GetCurrentLineNumber(ScannerRec* Scanner)
  217.     {
  218.         CheckPtrExistence(Scanner);
  219.         return Scanner->LineNumber;
  220.     }
  221.  
  222.  
  223. #define ENDOFTEXT (-1)
  224.  
  225. static int                    GetCharacter(ScannerRec* Scanner)
  226.     {
  227.         int                                Value;
  228.  
  229.         CheckPtrExistence(Scanner);
  230.         if (Scanner->Index == Scanner->BlockLength)
  231.             {
  232.                 return ENDOFTEXT;
  233.             }
  234.         Value = Scanner->Block[Scanner->Index];
  235.         Scanner->Index += 1;
  236.         return Value;
  237.     }
  238.  
  239.  
  240. static void                    UngetCharacter(ScannerRec* Scanner)
  241.     {
  242.         CheckPtrExistence(Scanner);
  243.         ERROR(Scanner->Index == 0,PRERR(ForceAbort,"UngetCharacter:  can't do it"));
  244.         Scanner->Index -= 1;
  245.     }
  246.  
  247.  
  248. /* get a token */
  249. TokenRec*                        GetNextToken(ScannerRec* Scanner)
  250.     {
  251.         int                                C;
  252.         TokenRec*                    Token;
  253.  
  254.         CheckPtrExistence(Scanner);
  255.  
  256.         /* check for pushback */
  257.         if (Scanner->PushedBackToken != NIL)
  258.             {
  259.                 TokenRec*                    Temp;
  260.  
  261.                 Temp = Scanner->PushedBackToken;
  262.                 Scanner->PushedBackToken = NIL;
  263.                 return Temp;
  264.             }
  265.  
  266.         /* get a character */
  267.         C = GetCharacter(Scanner);
  268.  
  269.         /* strip while space */
  270.         while (((C >= 0) && (C <= 32)) || (C == '#'))
  271.             {
  272.                 if ((C == 13) || (C == 10))
  273.                     {
  274.                         Scanner->LineNumber += 1;
  275.                     }
  276.                 if (C == '#')
  277.                     {
  278.                         /* comment */
  279.                         while ((C != 13) && (C != 10) && (C != ENDOFTEXT))
  280.                             {
  281.                                 C = GetCharacter(Scanner);
  282.                             }
  283.                     }
  284.                  else
  285.                     {
  286.                         C = GetCharacter(Scanner);
  287.                     }
  288.             }
  289.  
  290.         /* handle the end of text character */
  291.         if (C == ENDOFTEXT)
  292.             {
  293.                 Token = (TokenRec*)AllocTrackedBlock(sizeof(TokenRec),Scanner->Allocator);
  294.                 if (Token == NIL)
  295.                     {
  296.                         return NIL;
  297.                     }
  298.                 Token->Type = eTokenEndOfInput;
  299.             }
  300.  
  301.         /* handle a string literal */
  302.         else if (C == '\x22')
  303.             {
  304.                 StringThingRec*        String;
  305.                 char*                            StringCopy;
  306.                 long                            StringLength;
  307.  
  308.                 String = NewStringThing();
  309.                 if (String == NIL)
  310.                     {
  311.                         return NIL;
  312.                     }
  313.  
  314.                 C = GetCharacter(Scanner);
  315.                 while (C != '\x22')
  316.                     {
  317.                         char                            Buffer[1];
  318.  
  319.                         Buffer[0] = C;
  320.                         if (!AppendStringThing(String,&(Buffer[0]),1))
  321.                             {
  322.                                 DisposeStringThing(String);
  323.                                 return NIL;
  324.                             }
  325.                         if ((C == 10) || (C == 13))
  326.                             {
  327.                                 Scanner->LineNumber += 1;
  328.                             }
  329.                         C = GetCharacter(Scanner);
  330.                     }
  331.  
  332.                 StringCopy = StringThingGetSubrange(String,0,GetStringThingLength(String));
  333.                 DisposeStringThing(String);
  334.                 if (StringCopy == NIL)
  335.                     {
  336.                         return NIL;
  337.                     }
  338.                 StringLength = PtrSize(StringCopy);
  339.  
  340.                 Token = (TokenRec*)AllocTrackedBlock(sizeof(TokenRec),Scanner->Allocator);
  341.                 if (Token == NIL)
  342.                     {
  343.                         ReleasePtr(StringCopy);
  344.                         return NIL;
  345.                     }
  346.                 Token->Type = eTokenString;
  347.  
  348.                 Token->u.StringValue = AllocTrackedBlock(StringLength,Scanner->Allocator);
  349.                 if (Token->u.StringValue == NIL)
  350.                     {
  351.                         ReleasePtr(StringCopy);
  352.                         return NIL;
  353.                     }
  354.                 CopyData(StringCopy,Token->u.StringValue,StringLength);
  355.                 ReleasePtr(StringCopy);
  356.             }
  357.  
  358.         /* handle an identifier:  [a-zA-Z_][a-zA-Z0-9_]*  */
  359.         else if (((C >= 'a') && (C <= 'z')) || ((C >= 'A') && (C <= 'Z')) || (C == '_'))
  360.             {
  361.                 StringThingRec*        String;
  362.                 long                            LowBound;
  363.                 long                            HighBoundPlusOne;
  364.                 MyBoolean                    ContinueLoopingFlag;
  365.                 long                            KeywordIndex; /* -1 == not a keyword */
  366.                 char*                            StringCopy;
  367.                 long                            StringLength;
  368.  
  369.                 String = NewStringThing();
  370.                 if (String == NIL)
  371.                     {
  372.                         return NIL;
  373.                     }
  374.                 /* read the entire token */
  375.                 while (((C >= 'a') && (C <= 'z')) || ((C >= 'A') && (C <= 'Z')) || (C == '_')
  376.                     || ((C >= '0') && (C <= '9')))
  377.                     {
  378.                         char                            Buffer[1];
  379.  
  380.                         Buffer[0] = C;
  381.                         if (!AppendStringThing(String,&(Buffer[0]),1))
  382.                             {
  383.                                 DisposeStringThing(String);
  384.                                 return NIL;
  385.                             }
  386.                         C = GetCharacter(Scanner);
  387.                     }
  388.                 /* unget the character that made us stop */
  389.                 UngetCharacter(Scanner);
  390.                 /* get the string out of the line buffer */
  391.                 StringCopy = StringThingGetSubrange(String,0,GetStringThingLength(String));
  392.                 DisposeStringThing(String);
  393.                 if (StringCopy == NIL)
  394.                     {
  395.                         return NIL;
  396.                     }
  397.                 StringLength = PtrSize(StringCopy);
  398.  
  399.                 /* figure out if it is a keyword */
  400.                 LowBound = 0;
  401.                 HighBoundPlusOne = Scanner->NumKeywords;
  402.                 ContinueLoopingFlag = True;
  403.                 while (ContinueLoopingFlag)
  404.                     {
  405.                         long                            MidPoint;
  406.                         int                                CompareResult;
  407.  
  408.                         ERROR(LowBound > HighBoundPlusOne,PRERR(ForceAbort,
  409.                             "GetNextToken:  we seem to have a problem with low and high bounds"));
  410.  
  411.                         MidPoint = (LowBound + HighBoundPlusOne) / 2;
  412.  
  413.                         CompareResult = CompareStrings(StringCopy,PtrSize(StringCopy),
  414.                             Scanner->KeywordList[MidPoint].KeywordName,
  415.                             Scanner->KeywordList[MidPoint].KeywordLength);
  416.                         /* CompareResult == 0  -->  found the target */
  417.                         /* CompareResult < 0  --> in the first half of the list */
  418.                         /* CompareResult > 0  --> in the second half of the list */
  419.  
  420.                         if (CompareResult == 0)
  421.                             {
  422.                                 /* found the one */
  423.                                 KeywordIndex = MidPoint;
  424.                                 ContinueLoopingFlag = False;
  425.                             }
  426.                          else
  427.                             {
  428.                                 if (CompareResult < 0)
  429.                                     {
  430.                                         /* select first half of list */
  431.                                         HighBoundPlusOne = MidPoint;
  432.                                     }
  433.                                 else /* if (CompareResult > 0) */
  434.                                     {
  435.                                         /* select second half of list */
  436.                                         LowBound = MidPoint + 1;
  437.                                     }
  438.                                 /* termination condition:  if range in array collapses to an */
  439.                                 /* empty array, then there is no entry in the array */
  440.                                 if (LowBound == HighBoundPlusOne)
  441.                                     {
  442.                                         KeywordIndex = -1; /* indicate there is no keyword */
  443.                                         ContinueLoopingFlag = False;
  444.                                     }
  445.                             }
  446.                     }
  447.  
  448.                 /* create the token */
  449.                 Token = (TokenRec*)AllocTrackedBlock(sizeof(TokenRec),Scanner->Allocator);
  450.                 if (Token == NIL)
  451.                     {
  452.                         ReleasePtr(StringCopy);
  453.                         return NIL;
  454.                     }
  455.  
  456.                 if (KeywordIndex == -1)
  457.                     {
  458.                         /* no keyword; make a string containing token */
  459.                         Token->u.StringValue = AllocTrackedBlock(StringLength,Scanner->Allocator);
  460.                         if (Token->u.StringValue == NIL)
  461.                             {
  462.                                 ReleasePtr(StringCopy);
  463.                                 return NIL;
  464.                             }
  465.                         CopyData(StringCopy,Token->u.StringValue,StringLength);
  466.                         Token->Type = eTokenIdentifier;
  467.                     }
  468.                  else
  469.                     {
  470.                         Token->Type = eTokenKeyword;
  471.                         Token->u.IntegerValue = Scanner->KeywordList[KeywordIndex].TagValue;
  472.                     }
  473.                 ReleasePtr(StringCopy);
  474.             }
  475.  
  476.         /* integer or floating?  [0-9]+  [0-9]+"."[0-9]+([Ee][+-]?[0-9]+)?[sdf]?  */
  477.         else if (((C >= '0') && (C <= '9')) || (C == '.'))
  478.             {
  479.                 NumFormType                SpecifiedNumType;
  480.                 NumStateType            NumberState;
  481.                 StringThingRec*        String;
  482.                 char*                            StringData;
  483.                 long                            StringLength;
  484.  
  485.                 /* initialize the state value */
  486.                 NumberState = eIntegerPart;
  487.                 SpecifiedNumType = eTypeNotSpecified;
  488.  
  489.                 Token = (TokenRec*)AllocTrackedBlock(sizeof(TokenRec),Scanner->Allocator);
  490.                 if (Token == NIL)
  491.                     {
  492.                         return NIL;
  493.                     }
  494.  
  495.                 String = NewStringThing();
  496.                 if (String == NIL)
  497.                     {
  498.                         return NIL;
  499.                     }
  500.  
  501.                 while (((C >= '0') && (C <= '9')) || (C == '.') || (C == '+') || (C == '-')
  502.                     || (C == 's') || (C == 'd') || (C == 'f') || (C == 'e') || (C == 'E'))
  503.                     {
  504.                         char                            Buffer[1];
  505.  
  506.                         /* do some state changes */
  507.                         if (C == '.')
  508.                             {
  509.                                 if (NumberState != eIntegerPart)
  510.                                     {
  511.                                         Token->Type = eTokenError;
  512.                                         Token->u.IntegerValue = eScannerMalformedFloat;
  513.                                         DisposeStringThing(String);
  514.                                         goto AbortNumberErrorPoint;
  515.                                     }
  516.                                  else
  517.                                     {
  518.                                         NumberState = eFractionalPart;
  519.                                     }
  520.                             }
  521.                         else if ((C == 'e') || (C == 'E'))
  522.                             {
  523.                                 if ((NumberState != eIntegerPart) && (NumberState != eFractionalPart))
  524.                                     {
  525.                                         Token->Type = eTokenError;
  526.                                         Token->u.IntegerValue = eScannerMalformedFloat;
  527.                                         DisposeStringThing(String);
  528.                                         goto AbortNumberErrorPoint;
  529.                                     }
  530.                                  else
  531.                                     {
  532.                                         NumberState = eExponentialPart;
  533.                                     }
  534.                             }
  535.                         else if ((C == '+') || (C == '-'))
  536.                             {
  537.                                 if (NumberState != eExponentialPart)
  538.                                     {
  539.                                         /* this is not an error, since it could be a unary operator */
  540.                                         /* coming later, so we stop, but don't abort */
  541.                                         goto FinishNumberPoint; /* character ungot at target */
  542.                                     }
  543.                                  else
  544.                                     {
  545.                                         NumberState = eExponNumberPart;
  546.                                     }
  547.                             }
  548.                         else if (C == 's')
  549.                             {
  550.                                 if (NumberState == eNumberFinished)
  551.                                     {
  552.                                         Token->Type = eTokenError;
  553.                                         Token->u.IntegerValue = eScannerMalformedFloat;
  554.                                         DisposeStringThing(String);
  555.                                         goto AbortNumberErrorPoint;
  556.                                     }
  557.                                  else
  558.                                     {
  559.                                         NumberState = eNumberFinished;
  560.                                         SpecifiedNumType = eTypeSingle;
  561.                                         C = 32; /* so adding it to the string doesn't do damage */
  562.                                     }
  563.                             }
  564.                         else if (C == 'd')
  565.                             {
  566.                                 if (NumberState == eNumberFinished)
  567.                                     {
  568.                                         Token->Type = eTokenError;
  569.                                         Token->u.IntegerValue = eScannerMalformedFloat;
  570.                                         DisposeStringThing(String);
  571.                                         goto AbortNumberErrorPoint;
  572.                                     }
  573.                                  else
  574.                                     {
  575.                                         NumberState = eNumberFinished;
  576.                                         SpecifiedNumType = eTypeDouble;
  577.                                         C = 32;
  578.                                     }
  579.                             }
  580.                         else if (C == 'f')
  581.                             {
  582.                                 if (NumberState == eNumberFinished)
  583.                                     {
  584.                                         Token->Type = eTokenError;
  585.                                         Token->u.IntegerValue = eScannerMalformedFloat;
  586.                                         DisposeStringThing(String);
  587.                                         goto AbortNumberErrorPoint;
  588.                                     }
  589.                                  else
  590.                                     {
  591.                                         NumberState = eNumberFinished;
  592.                                         SpecifiedNumType = eTypeFixed;
  593.                                         C = 32;
  594.                                     }
  595.                             }
  596.  
  597.                         /* actually save the character */
  598.                         Buffer[0] = C;
  599.                         if (!AppendStringThing(String,&(Buffer[0]),1))
  600.                             {
  601.                                 DisposeStringThing(String);
  602.                                 return NIL;
  603.                             }
  604.  
  605.                         C = GetCharacter(Scanner);
  606.                     }
  607.              FinishNumberPoint:
  608.                 UngetCharacter(Scanner);
  609.  
  610.                 StringData = StringThingGetSubrange(String,0,GetStringThingLength(String));
  611.                 DisposeStringThing(String);
  612.                 if (StringData == NIL)
  613.                     {
  614.                         return NIL;
  615.                     }
  616.                 StringLength = PtrSize(StringData);
  617.  
  618.                 /* if the token type is not specified, then see what we can guess */
  619.                 if (eTypeNotSpecified == SpecifiedNumType)
  620.                     {
  621.                         if (NumberState == eIntegerPart)
  622.                             {
  623.                                 /* if we only got as far as the integer part, then it's an int */
  624.                                 SpecifiedNumType = eTypeInteger;
  625.                             }
  626.                          else
  627.                             {
  628.                                 /* otherwise, assume the highest precision type */
  629.                                 SpecifiedNumType = eTypeDouble;
  630.                             }
  631.                     }
  632.  
  633.                 /* create the token */
  634.                 switch (SpecifiedNumType)
  635.                     {
  636.                         default:
  637.                             EXECUTE(PRERR(ForceAbort,"GetNextToken:  bad number type specifier"));
  638.                             break;
  639.                         case eTypeSingle:
  640.                             Token->Type = eTokenSingle;
  641.                             Token->u.SingleValue = StringToLongDouble(StringData,StringLength);
  642.                             break;
  643.                         case eTypeDouble:
  644.                             Token->Type = eTokenDouble;
  645.                             Token->u.DoubleValue = StringToLongDouble(StringData,StringLength);
  646.                             break;
  647.                         case eTypeFixed:
  648.                             Token->Type = eTokenFixed;
  649.                             Token->u.FixedValue = double2largefixed(StringToLongDouble(StringData,
  650.                                 StringLength));
  651.                             break;
  652.                         case eTypeInteger:
  653.                             Token->Type = eTokenInteger;
  654.                             Token->u.IntegerValue = StringToInteger(StringData,StringLength);
  655.                             break;
  656.                     }
  657.                 ReleasePtr(StringData);
  658.  
  659.                 /* this is the escape point for when a bad character is encountered. */
  660.              AbortNumberErrorPoint:
  661.                 ;
  662.             }
  663.  
  664.         /* handle a symbol */
  665.         else
  666.             {
  667.                 Token = (TokenRec*)AllocTrackedBlock(sizeof(TokenRec),Scanner->Allocator);
  668.                 if (Token == NIL)
  669.                     {
  670.                         return NIL;
  671.                     }
  672.  
  673.                 switch (C)
  674.                     {
  675.                         default:
  676.                             Token->Type = eTokenError;
  677.                             Token->u.IntegerValue = eScannerUnknownCharacter;
  678.                             break;
  679.                         case OPAREN:
  680.                             Token->Type = eTokenOpenParen;
  681.                             break;
  682.                         case CPAREN:
  683.                             Token->Type = eTokenCloseParen;
  684.                             break;
  685.                         case OBRACKET:
  686.                             Token->Type = eTokenOpenBracket;
  687.                             break;
  688.                         case CBRACKET:
  689.                             Token->Type = eTokenCloseBracket;
  690.                             break;
  691.                         case ':':
  692.                             C = GetCharacter(Scanner);
  693.                             if (C == '=')
  694.                                 {
  695.                                     Token->Type = eTokenColonEqual;
  696.                                 }
  697.                              else
  698.                                 {
  699.                                     /* push the character back */
  700.                                     UngetCharacter(Scanner);
  701.                                     Token->Type = eTokenColon;
  702.                                 }
  703.                             break;
  704.                         case ';':
  705.                             Token->Type = eTokenSemicolon;
  706.                             break;
  707.                         case ',':
  708.                             Token->Type = eTokenComma;
  709.                             break;
  710.                         case '+':
  711.                             Token->Type = eTokenPlus;
  712.                             break;
  713.                         case '-':
  714.                             Token->Type = eTokenMinus;
  715.                             break;
  716.                         case '*':
  717.                             Token->Type = eTokenStar;
  718.                             break;
  719.                         case '/':
  720.                             Token->Type = eTokenSlash;
  721.                             break;
  722.                         case '=':
  723.                             Token->Type = eTokenEqual;
  724.                             break;
  725.                         case '<':
  726.                             C = GetCharacter(Scanner);
  727.                             if (C == '>')
  728.                                 {
  729.                                     Token->Type = eTokenNotEqual;
  730.                                 }
  731.                             else if (C == '=')
  732.                                 {
  733.                                     Token->Type = eTokenLessThanOrEqual;
  734.                                 }
  735.                             else if (C == '<')
  736.                                 {
  737.                                     Token->Type = eTokenShiftLeft;
  738.                                 }
  739.                             else
  740.                                 {
  741.                                     Token->Type = eTokenLessThan;
  742.                                     UngetCharacter(Scanner);
  743.                                 }
  744.                             break;
  745.                         case '>':
  746.                             C = GetCharacter(Scanner);
  747.                             if (C == '=')
  748.                                 {
  749.                                     Token->Type = eTokenGreaterThanOrEqual;
  750.                                 }
  751.                             else if (C == '>')
  752.                                 {
  753.                                     Token->Type = eTokenShiftRight;
  754.                                 }
  755.                             else
  756.                                 {
  757.                                     Token->Type = eTokenGreaterThan;
  758.                                     UngetCharacter(Scanner);
  759.                                 }
  760.                             break;
  761.                         case '^':
  762.                             Token->Type = eTokenCircumflex;
  763.                             break;
  764.                     }
  765.             }
  766.  
  767.         return Token;
  768.     }
  769.  
  770.  
  771. /* push a token back onto the token stream */
  772. void                                UngetToken(ScannerRec* Scanner, TokenRec* Token)
  773.     {
  774.         CheckPtrExistence(Scanner);
  775.         CheckPtrExistence(Token);
  776.         ERROR(Scanner->PushedBackToken != NIL,PRERR(ForceAbort,
  777.             "UngetToken:  there is already a token pushed back"));
  778.         Scanner->PushedBackToken = Token;
  779.     }
  780.  
  781.  
  782. /* get the type of a token */
  783. TokenTypes                    GetTokenType(TokenRec* Token)
  784.     {
  785.         CheckPtrExistence(Token);
  786.         return Token->Type;
  787.     }
  788.  
  789.  
  790. /* get the string associated with an identifier */
  791. char*                                GetTokenIdentifierString(TokenRec* Token)
  792.     {
  793.         CheckPtrExistence(Token);
  794.         ERROR(Token->Type != eTokenIdentifier,PRERR(ForceAbort,
  795.             "GetTokenIdentifierString:  token isn't an identifier"));
  796.         return Token->u.StringValue;
  797.     }
  798.  
  799.  
  800. /* get the string associated with a string token */
  801. char*                                GetTokenStringValue(TokenRec* Token)
  802.     {
  803.         CheckPtrExistence(Token);
  804.         ERROR(Token->Type != eTokenString,PRERR(ForceAbort,
  805.             "GetTokenStringValue:  token isn't a string"));
  806.         return Token->u.StringValue;
  807.     }
  808.  
  809.  
  810. /* get the integer value associated with an integer token */
  811. long                                GetTokenIntegerValue(TokenRec* Token)
  812.     {
  813.         CheckPtrExistence(Token);
  814.         ERROR(Token->Type != eTokenInteger,PRERR(ForceAbort,
  815.             "GetTokenIntegerValue:  token isn't an integer"));
  816.         return Token->u.IntegerValue;
  817.     }
  818.  
  819.  
  820. /* get the single precision value associated with a single float token */
  821. float                                GetTokenSingleValue(TokenRec* Token)
  822.     {
  823.         CheckPtrExistence(Token);
  824.         ERROR(Token->Type != eTokenSingle,PRERR(ForceAbort,
  825.             "GetTokenSingleValue:  token isn't a single"));
  826.         return Token->u.SingleValue;
  827.     }
  828.  
  829.  
  830. /* get the double precision value associated with a double float token */
  831. double                            GetTokenDoubleValue(TokenRec* Token)
  832.     {
  833.         CheckPtrExistence(Token);
  834.         ERROR(Token->Type != eTokenDouble,PRERR(ForceAbort,
  835.             "GetTokenDoubleValue:  token isn't a double"));
  836.         return Token->u.DoubleValue;
  837.     }
  838.  
  839.  
  840. /* get the fixed precision value associated with a fixed token */
  841. largefixedsigned        GetTokenFixedValue(TokenRec* Token)
  842.     {
  843.         CheckPtrExistence(Token);
  844.         ERROR(Token->Type != eTokenFixed,PRERR(ForceAbort,
  845.             "GetTokenFixedValue:  token isn't a fixed point"));
  846.         return Token->u.FixedValue;
  847.     }
  848.  
  849.  
  850. /* get the tag associated with a keyword token */
  851. long                                GetTokenKeywordTag(TokenRec* Token)
  852.     {
  853.         CheckPtrExistence(Token);
  854.         ERROR(Token->Type != eTokenKeyword,PRERR(ForceAbort,
  855.             "GetTokenKeywordTag:  token isn't a keyword"));
  856.         return Token->u.IntegerValue;
  857.     }
  858.  
  859.  
  860. /* get the error code associated with an error token */
  861. ScannerErrors                GetTokenErrorCode(TokenRec* Token)
  862.     {
  863.         CheckPtrExistence(Token);
  864.         ERROR(Token->Type != eTokenError,PRERR(ForceAbort,
  865.             "GetTokenErrorCode:  token isn't a keyword"));
  866.         return (ScannerErrors)Token->u.IntegerValue;
  867.     }
  868.